const fs = require('fs');
const Papa = require('papaparse');

// 使用PapaParse库处理大量CSV数据
const startTime = Date.now();
let recordCount = 0;
let sampleRecords = [];

// 创建可读流
const fileStream = fs.createReadStream('../521.CSV', { encoding: 'utf8' });

// 使用PapaParse的流式处理
Papa.parse(fileStream, {
  header: true,
  dynamicTyping: true, // 自动转换类型
  skipEmptyLines: true,
  trimHeaders: true,
  fastMode: true, // 快速模式，适用于没有引号的CSV
  chunk: function(results, parser) {
    // 分块处理数据
    recordCount += results.data.length;
    
    // 保存前3条记录作为示例
    if (sampleRecords.length < 3 && results.data.length > 0) {
      const neededSamples = Math.min(3 - sampleRecords.length, results.data.length);
      sampleRecords = sampleRecords.concat(results.data.slice(0, neededSamples));
    }
    
    // 每处理10000行输出一次进度
    if (recordCount % 10000 === 0) {
      console.log(`已处理 ${recordCount} 行数据...`);
    }
  },
  complete: function() {
    const endTime = Date.now();
    const duration = (endTime - startTime) / 1000;
    
    console.log(`成功解析记录数: ${recordCount}`);
    console.log(`解析耗时: ${duration} 秒`);
    console.log('前3条记录示例:');
    console.log(sampleRecords);
  },
  error: function(error) {
    console.error('解析错误:', error.message);
  }
});